/*
 * Copyright (c) 2018-2024 北京华恒盛世科技有限公司
 * QUICKPOOL is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *     http://license.coscl.org.cn/MulanPSL2
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
*/

#include "subcmd.h"

#define MAX_TIMERSTRLEN         20
#define JOBID_F 7
#define USER_F 10
#define STAT_F 5
#define QUEUE_F 10
#define SUBMIT_NODE_F 11
#define EXEC_NODE_F 10
#define JOBNAME 10
#define TIME_F 20


static json *jobinfo_2json(struct QuickPoolJobInfo *, struct QuickPoolJobReply *reply,
                           struct quickpoolInfo *);

static bool_t isquickpool_admin(void);
static void listjob_usage(const char *);

/**
 * @brief       qp job list handler, to query job information from js.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 *
 * @retval      0             #1: succeed
 * @retval      1             #2: input option is wrong or kind of input argument not specified
 * @retval      2             #3: argument has syntax error
 * @retval      3             #4: failed to query job information
 * @retval      4             #5: not enough memory to print json string
 */
int
cmd_list_jobs(int argc, char **argv)
{
    int options=CUR_JOB;
    char *strUser=NULL, *strQueue=NULL, *strNode=NULL, *strJobName=NULL;
    char *strProject=NULL, *strJobGroup=NULL;
    bool_t fOpt=FALSE;
    bool_t wOpt=FALSE;
    while (TRUE) {
        int option_index = 0;
        static struct option long_options[] = {
            {"help", no_argument, 0, 'h'},
            {"array", no_argument, 0, 'A'},
            {"all", no_argument, 0, 'a'},
            {"done", no_argument, 0, 'd'},
            {"exit", no_argument, 0, 'e'},
            {"pend", no_argument, 0, 'p'},
            {"suspend", no_argument, 0, 's'},
            {"start", no_argument, 0, 'r'},
            {"queue", required_argument, 0, 'q'},
            {"node", required_argument, 0, 'm'},
            {"user", required_argument, 0, 'u'},
            {"jobname", required_argument, 0, 'J'},
            {"project", required_argument, 0, 'P'},
            {"jobgroup", required_argument, 0, 'g'},
            {"format", no_argument, 0, 'f'},
            {"wide", no_argument, 0, 'w'},
            {"gpu", no_argument, 0, 'G'},
            {NULL, 0, 0, 0}
        };

        int opt;
        opt = getopt_long(argc, argv, "wfhAadepsrq:m:u:J:P:g:G", long_options, &option_index);
        if (-1 == opt) {
            break;
        }

        switch (opt) {
        case 'A':
            options |= JGRP_ARRAY_INFO;
            options &= ~CUR_JOB;
            break;
        case 'a':
            options |= ALL_JOB;
            break;
        case 'd':
            options |= DONE_JOB;
            options &= ~CUR_JOB;
            break;
        case 'e':
            options |= EXIT_JOB;
            options &= ~CUR_JOB;
            break;
        case 'p':
            options |= PEND_JOB;
            options &= ~CUR_JOB;
            break;
        case 's':
            options |= SUSP_JOB;
            options &= ~CUR_JOB;
            break;
        case 'r':
            options |= RUN_JOB;
            options &= ~CUR_JOB;
            break;
        case 'f':
            fOpt = TRUE;
            break;
        case 'w':
            wOpt = TRUE;
            break;
        case 'q':
            if (NULL != strQueue) {
                fprintf(stderr, "Do not specify -q/--queue option for multiple times.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Must specify a queue name with -q/--queue option.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            strQueue = optarg;
            break;
        case 'u':
            if (NULL != strUser) {
                fprintf(stderr, "Do not specify -u/--user option for multiple times.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Specify a user name with -u/--user option.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            strUser = optarg;
            break;
        case 'm':
            if (NULL != strNode) {
                fprintf(stderr, "Do not specify -m/--node option for multiple times.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Must specify a node name with -m/--node option.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            strNode = optarg;
            break;
        case 'J':
            if (NULL != strJobName) {
                fprintf(stderr, "Do not specify -J/--jobname option for multiple times.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Must specify a job name with -J/--jobname option.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            strJobName = optarg;
            break;
        case 'P':
            if (NULL != strProject) {
                fprintf(stderr, "Do not specify -P/--project option for multiple times.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Must specify a project name with -P/--project option.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            strProject = optarg;
            break;
        case 'g':
            if (NULL != strJobGroup) {
                fprintf(stderr, "Do not specify -g/--jobgroup option for multiple times.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            if (0 >= strlen(optarg)) {
                fprintf(stderr, "Must specify a job group path with -g/--jobgroup option.\n");
                listjob_usage(argv[0]);
                exit(1);
            }

            strJobGroup = optarg;
            break;
        case 'G':
            options |= GPU_JOB;
            break;
        case 'h':
            listjob_usage(argv[0]);
            exit(0);
        case '?':
        default:
            listjob_usage(argv[0]);
            exit(1);
        }
    }

    unsigned int numJobIds;
    JOBID_T *specJobIds = NULL;
    numJobIds = get_jobids_fromargv(argc, argv, &specJobIds);
    if (0 < numJobIds) {
        strUser = ALL_STRING;
        options |= ALL_JOB;
    } else {
        if (NULL == strUser) {
            if ((getuid() == 0) || isquickpool_admin()) {
                strUser = ALL_STRING;
            }
        }
    }

    options |= HOST_NAME; // tell JS to reply host names

    if ((options & JGRP_ARRAY_INFO) && numJobIds <= 0) {
        if (NULL == strJobName) {
            strJobName = (char *)"*";
        }
    }

    struct quickpoolInfo *qpInfo;
    qpInfo = qp_info(NULL);
    if (NULL == qpInfo) {
        exit(3);
    }

    unsigned int num = 2; //options, username
    if (0 < numJobIds) {
        num++;
    } else {

        if (NULL != strQueue) {
            num++;
        }

        if (NULL != strNode) {
            num++;
        }

        if (NULL != strJobName) {
            num++;
        }

        if (NULL != strProject) {
            num++;
        }

        if (NULL != strJobGroup) {
            num++;
        }
    }

    struct request jobRequest;
    memset(&jobRequest, 0, sizeof(struct request));

    jobRequest.number = num;
    jobRequest.keyvalues = (struct keyvalue *)calloc(jobRequest.number, sizeof(struct keyvalue));
    if (NULL == jobRequest.keyvalues) {
        fprintf(stderr, "calloc(%ld) failed due to %m.\n",
                jobRequest.number*sizeof(struct keyvalue));
        exit(4);
    }

    num = 0;

    jobRequest.keyvalues[num].key = REQUEST_KEY_OPTIONS;
    jobRequest.keyvalues[num].value = (char *)calloc(12, sizeof(char));
    if (NULL == jobRequest.keyvalues[num].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
        exit(4);
    }
    snprintf(jobRequest.keyvalues[num].value, 12, "%d", options);
    num++;

    jobRequest.keyvalues[num].key = REQUEST_KEY_USERNAME;
    if (NULL == strUser) {
        jobRequest.keyvalues[num].value = getmy_uname();
    } else {
        jobRequest.keyvalues[num].value = copy_string(strUser);
    }
    num ++;

    unsigned int i;
    if (0 < numJobIds) {

        // to query specified job information.
        jobRequest.keyvalues[num].key = REQUEST_KEY_JOBIDS;
        unsigned int size = numJobIds*sizeof(JOBID_T) + 10;
        jobRequest.keyvalues[num].value = (char *)calloc(size+1, sizeof(char));
        if (NULL == jobRequest.keyvalues[num].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", size);
            exit(4);
        }

        for (i=0; i<numJobIds; i++) {
            if (0 == i) {
                snprintf(jobRequest.keyvalues[num].value, size, "%lld", specJobIds[i]);
                continue;
            }

            unsigned int lenValue;
            lenValue = strlen(jobRequest.keyvalues[num].value);
            snprintf(jobRequest.keyvalues[num].value+lenValue, size, " %lld", specJobIds[i]);
        }

        FREEUP(specJobIds);
        num ++;
        // ignore all other options when job ids are specified.
    } else {

        if (NULL != strQueue) { // to query job informatio of specified queue.
            jobRequest.keyvalues[num].key = REQUEST_KEY_QUEUENAME;
            jobRequest.keyvalues[num].value = copy_string(strQueue);
            num ++;
        }

        if (NULL != strNode) { // to query job informatio which is running on specified node.
            jobRequest.keyvalues[num].key = REQUEST_KEY_HOSTNAME;
            jobRequest.keyvalues[num].value = copy_string(strNode);
            num ++;
        }

        if (NULL != strJobName) { // to query job informatio of specified job name.
            jobRequest.keyvalues[num].key = REQUEST_KEY_JOBNAME;
            jobRequest.keyvalues[num].value = copy_string(strJobName);
            num ++;
        }

        if (NULL != strProject) { // to query job informatio of specified project.
            jobRequest.keyvalues[num].key = REQUEST_KEY_SUBMIT_PROJECT;
            jobRequest.keyvalues[num].value = copy_string(strProject);
            num ++;
        }

        if (NULL != strJobGroup) { // to query job informatio of specified job group.
            jobRequest.keyvalues[num].key = REQUEST_KEY_SUBMIT_JOBGROUP;
            jobRequest.keyvalues[num].value = copy_string(strJobGroup);
            num ++;
        }
    }

    // first, we got all job IDs of the request from JS.
    struct QuickPoolJobReply *jobReply;
    jobReply = client_get_jobids(&jobRequest, NULL);
    if (NULL == jobReply) {

        fprintf(stderr, "Faild to get job information due to above reason.\n");
        gf_xdr_free(gf_xdr_request, &jobRequest);

        exit(3);
    }
    
    gf_xdr_free(gf_xdr_request, &jobRequest);

    json *jsonJobList = json_CreateObject();
    if ((fOpt | wOpt) && (0 < jobReply->num_job)) {
        char buf[4096+1];

        snprintf(buf, 4096, "%%-%ds  %%-%ds  %%-%ds  %%-%ds  %%-%ds  %%-%ds  %%-%ds  %%-%ds\n",
                 JOBID_F, USER_F, STAT_F, QUEUE_F, SUBMIT_NODE_F, EXEC_NODE_F, JOBNAME, TIME_F);
        fprintf(stdout,buf, "ID", "USER", "STAT", "QUEUE", "SUBMIT_NODE", "EXEC_NODE", "JOB_NAME", "SUBMIT_TIME");

        for (i = 0; i < jobReply->num_job; i++) {
            struct QuickPoolJobInfo *job;
            job = client_get_jobinfo();
            if (NULL == job) {
            exit(3);
            }
            char strJobId[32];
            char timebuf[32];
            const char *strstatus;
            char strExec_node[PATH_MAX+1];
            if (job->num_execute_node == 0) {
                strncpy(strExec_node, " ", PATH_MAX);
            } else {
                snprintf(strExec_node, PATH_MAX, "%d*%s", *job->num_slot, *job->execute_nodes);
            }
            strstatus = get_status(job);
            ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &job->submit->submit_time, timebuf);
            jobid_2string(job->jobid,strJobId);
            if (wOpt) {

                fprintf(stdout, buf,strJobId, job->submit->user, strstatus, job->submit->queue,
                                                    job->submit->from_node, strExec_node, job->submit->job_name, timebuf);
                continue;
            }
            if (strlen(strJobId) > JOBID_F) {
                strJobId[JOBID_F-1] = '*';
                strJobId[JOBID_F] = '\0';
            }
            if (strlen(job->submit->user) > USER_F) {
                job->submit->user[USER_F-1] = '*';
                job->submit->user[USER_F] = '\0';
            }
            if (strlen(job->submit->queue) > QUEUE_F) {
                job->submit->queue[QUEUE_F-1] = '*';
                job->submit->queue[QUEUE_F] = '\0';
            }
            if (strlen(job->submit->from_node) > SUBMIT_NODE_F) {
                job->submit->from_node[SUBMIT_NODE_F-1] = '*';
                job->submit->from_node[SUBMIT_NODE_F] = '\0';
            }
            if (strlen(strExec_node) > EXEC_NODE_F) {
                strExec_node[EXEC_NODE_F-1] = '*';
                strExec_node[EXEC_NODE_F] = '\0';
            }
            if (strlen(job->submit->job_name) > JOBNAME) {
                job->submit->job_name[JOBNAME-1] = '*';
                job->submit->job_name[JOBNAME] = '\0';
            }
            fprintf(stdout, buf, strJobId, job->submit->user, strstatus, job->submit->queue,
                                                    job->submit->from_node, strExec_node, job->submit->job_name, timebuf);
            FREEUP(job);
        }
        
        FREEUP(jobReply);
        exit(0);
    }
    
    // get job information from JS one by one.
    for (i = 0; i < jobReply->num_job; i++) {
        struct QuickPoolJobInfo *job;

        job = client_get_jobinfo();
        if (NULL == job) {
            exit(3);
        }

        json *jsonJob = jobinfo_2json(job, jobReply, qpInfo);
        char strJobId[32];
        jobid_2string(job->jobid, strJobId);
        json_AddItemToObject(jsonJobList, strJobId, jsonJob);

        gf_xdr_free(libxdr_jobinfo, job);
        FREEUP(job);
    }
    
    // done, close the connection with JS.
    client_finish_getting();

    if (0 < jobReply->num_job) {

        char *strJson;
        strJson = json_Print(jsonJobList);
        if (NULL != strJson) {
            fprintf(stdout, "%s\n", strJson);
            FREEUP(strJson);
        } else {

            perror("malloc");
            json_Delete(jsonJobList);
            gf_xdr_free(libxdr_job_reply, jobReply);
            FREEUP(jobReply);

            exit(4);
        }
    }
    json_Delete(jsonJobList);

    for (i = 0; i < jobReply->num_error; i++) {
        fprintf(stderr, "%s.\n", jobReply->error_messages[i]);
    }

    if (0 < jobReply->num_error) {
        gf_xdr_free(libxdr_job_reply, jobReply);
        FREEUP(jobReply);
        exit(3);
    }

    gf_xdr_free(libxdr_job_reply, jobReply);
    FREEUP(jobReply);

    exit(0);
} // end function cmd_list_jobs

/* ------------------------------------------------
 *
 *  below are static functions used in this file
 *
 * ------------------------------------------------ */

/**
 * @brief       transform job information to json.
 *
 * @param[in]   job           #1: job being tranformed
 * @param[in]   reply         #2: job reply header, pending reason display its node names
 * @param[in]   info          #3: quickpool information, pending reason
 *                                display its resource names
 *
 * @retval      not NULL      #1: transformed json object
 */
static json *
jobinfo_2json(struct QuickPoolJobInfo *job, struct QuickPoolJobReply *reply,
              struct quickpoolInfo *info)
{
    json *jsonJob = json_CreateObject();

    struct submitRequest *submitInfo = job->submit;

    json_AddNumberToObject(jsonJob, "JOBID", ARRAY_JOBID(job->jobid));
    if (0 < ARRAY_INDEX(job->jobid)) {
        json_AddNumberToObject(jsonJob, "JOB_INDEX", ARRAY_INDEX(job->jobid));
    }

    const char *strStatus;
    strStatus = get_status(job);
    json_AddStringToObject(jsonJob, "STATUS", strStatus);

    strStatus = get_post_status(job);
    if (NULL != strStatus) {
        json_AddStringToObject(jsonJob, "POST_STATUS", strStatus);
    }

    if (0 < job->job_priority) {
        s2j_json_set_basic_element(jsonJob, job, int, "JOB_PRIORITY", job_priority);
    }

    json *jsonJobSubmit = submitrequest_2json(job);
    json_AddItemToObject(jsonJob, "SUBMIT_REQUEST", jsonJobSubmit);

    if (0 < job->num_pending_reason) {
        char *pendReasons;
        switch (job->status) {
        case JOB_STATUS_PSUSP:
        case JOB_STATUS_PEND:

            pendReasons = get_pending_reason(job->num_pending_reason, job->pending_reasons,
                                             reply, info->resource_table);
            json_AddStringToObject(jsonJob, "PENDING_REASON", pendReasons);
            break;
        }
    }

    char timebuf[TIMESTRINGLEN];
    unsigned int i;
    if (0 < job->num_execute_node && 0 < job->reserving_time) {

        json *jsonJobReservation = json_CreateObject();

        ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &job->reserving_time, timebuf);
        json_AddStringToObject(jsonJobReservation, "RESERVE_TIME", timebuf);

        int numRsvSlot=0;
        json *jsonNodeArray = json_CreateArray();
        for (i=0; i<job->num_execute_node; i++) {
            char strExecNode[PATH_MAX+1];

            snprintf(strExecNode, PATH_MAX, "%d*%s", job->num_slot[i], job->execute_nodes[i]);
	    json_AddItemToArray(jsonNodeArray, json_CreateString(strExecNode));
            numRsvSlot += job->num_slot[i];
        }
        json_AddNumberToObject(jsonJobReservation, "NUM_RESERVE_SLOT", numRsvSlot);
        json_AddNumberToObject(jsonJobReservation, "NUM_RESERVE_NODE", job->num_execute_node);
        json_AddItemToObject(jsonJobReservation, "RESERVE_NODE", jsonNodeArray);

        if (0 < job->possible_start && IS_JOB_PENDING(job->status)) {

            ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &job->possible_start, timebuf);
            json_AddStringToObject(jsonJobReservation, "WILL_START", timebuf);
        }

        json_AddItemToObject(jsonJob, "RESERVATION", jsonJobReservation);
    }

    json *jsonLoadSchedule;
    jsonLoadSchedule = json_CreateObject();

    json *jsonLoadStop;
    jsonLoadStop = json_CreateObject();

    for (i=0; i<info->num_index; i++) {
        char strLoadSchedule[64+1], strLoadStop[64+1];
        strLoadSchedule[0] = '\0';
        strLoadStop[0] = '\0';

        if (job->load_schedule[i] < INFINIT_LOAD && job->load_schedule[i] > -INFINIT_LOAD) {
            snprintf(strLoadSchedule, 64, "%.2f", job->load_schedule[i]);
        }

        if (job->load_stop[i] < INFINIT_LOAD && job->load_stop[i] > -INFINIT_LOAD) {
            snprintf(strLoadStop, 64, "%.2f", job->load_stop[i]);
        }

        if (0 < strlen(strLoadSchedule)) {
            json_AddStringToObject(jsonLoadSchedule, info->resource_table[i].name, strLoadSchedule);
        }

        if (0 < strlen(strLoadStop)) {
            json_AddStringToObject(jsonLoadStop, info->resource_table[i].name, strLoadStop);
        }
    }

    if (0 < json_GetArraySize(jsonLoadSchedule)) {
        json_AddItemToObject(jsonJob, "LOAD_SCHED", jsonLoadSchedule);
    } else {
        json_Delete(jsonLoadSchedule);
    }

    if (0 < json_GetArraySize(jsonLoadStop)) {
        json_AddItemToObject(jsonJob, "LOAD_STOP", jsonLoadStop);
    } else {
        json_Delete(jsonLoadStop);
    }

    if (job->job_type == JGRP_NODE_ARRAY) {
        json *jsonJobArray = json_CreateObject();

        json_AddNumberToObject(jsonJobArray, "JOB_ARRAY_ID", ARRAY_JOBID(job->jobid));
        json_AddStringToObject(jsonJobArray, "JOB_NAME", submitInfo->job_name);

        json_AddNumberToObject(jsonJobArray, "ARRAY_NJOBS", job->counter[JGRP_COUNT_NJOBS]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_PEND", job->counter[JGRP_COUNT_PEND]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_RUNNING", job->counter[JGRP_COUNT_NRUN]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_DONE", job->counter[JGRP_COUNT_NDONE]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_EXIT", job->counter[JGRP_COUNT_NEXIT]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_SSUSP", job->counter[JGRP_COUNT_NSSUSP]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_USUSP", job->counter[JGRP_COUNT_NUSUSP]);
        json_AddNumberToObject(jsonJobArray, "ARRAY_PSUSP", job->counter[JGRP_COUNT_NPSUSP]);

        json_AddItemToObject(jsonJob, "JOB_ARRAY_INFO", jsonJobArray);
    }

    if (0 < job->start_time && 0 < job->num_execute_node) {
        json *jsonJobStart = json_CreateObject();

        ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &(job->start_time), timebuf);
        json_AddStringToObject(jsonJobStart, "START_TIME", timebuf);

        json *jsonResourceUsage = json_CreateObject();

        long totalCpuTime=0, totalMem=0, totalSwap=0;
        int numAllocSlot=0;
        json *jsonNodeArray = json_CreateArray();
        for (i=0; i<job->num_execute_node; i++) {
            char strExecNode[PATH_MAX+1];

            snprintf(strExecNode, PATH_MAX, "%d*%s", job->num_slot[i], job->execute_nodes[i]);
	    json_AddItemToArray(jsonNodeArray, json_CreateString(strExecNode));
            numAllocSlot += job->num_slot[i];

            json *jsonUsageHost = json_CreateObject();
            if (0 < job->resource_usage[i].utime || 0 < job->resource_usage[i].stime) {
                char strCpuTime[24+1];

                snprintf(strCpuTime, 24, "%ld",
                         job->resource_usage[i].utime+job->resource_usage[i].stime);
                json_AddStringToObject(jsonUsageHost, "CPU_TIME", strCpuTime);

                totalCpuTime += job->resource_usage[i].utime+job->resource_usage[i].stime;
	    }

            if (0 < job->resource_usage[i].mem) {
                char *strUnit;
                float memVal;

                if (job->resource_usage[i].mem > 1024*1024*1024) {
                    memVal = (double)job->resource_usage[i].mem/(double)(1024*1024*1024);
                    strUnit = (char *)"T";
                } else if (job->resource_usage[i].mem > 1024*1024) {
                    memVal = (double)job->resource_usage[i].mem/(double)(1024*1024);
                    strUnit = (char *)"G";
                } else if (job->resource_usage[i].mem > 1024) {
                    memVal = (double)job->resource_usage[i].mem/(double)(1024);
                    strUnit = (char *)"M";
                } else {
                    memVal = job->resource_usage[i].mem;
                    strUnit = (char *)"K";
                }

                char strMemUsage[56+1];
                snprintf(strMemUsage, 56, "%.1f%s", memVal, strUnit);
                json_AddStringToObject(jsonUsageHost, "MEMORY_USAGE", strMemUsage);

                totalMem += job->resource_usage[i].mem;
            }

            if (0 < job->resource_usage[i].swap) {
                char *strUnit;
                float swapVal;

                if (job->resource_usage[i].swap > 1024*1024*1024) {
                    swapVal = (double)job->resource_usage[i].swap/(double)(1024*1024*1024);
                    strUnit = (char *)"T";
                } else if (job->resource_usage[i].swap > 1024*1024) {
                    swapVal = (double)job->resource_usage[i].swap/(double)(1024*1024);
                    strUnit = (char *)"G";
                } else if (job->resource_usage[i].swap > 1024) {
                    swapVal = (double)job->resource_usage[i].swap/(double)1024;
                    strUnit = (char *)"M";
                } else {
                    swapVal = job->resource_usage[i].swap;
                    strUnit = (char *)"K";
                }

                char strSwapUsage[56+1];
                snprintf(strSwapUsage, 56, "%.1f%s", swapVal, strUnit);
                json_AddStringToObject(jsonUsageHost, "SWAP_USAGE", strSwapUsage);

                totalSwap += job->resource_usage[i].swap;
            }

            if (0 < job->resource_usage[i].npgids) {
                s2j_json_set_array_element(jsonUsageHost, &(job->resource_usage[i]),
                                           int, "PGID", pgid, job->resource_usage[i].npgids);
            }

            json *jsonPidArray;
            if (0 < job->resource_usage[i].npids) {
                jsonPidArray = json_CreateArray();
            }
            unsigned int j;
            for (j=0; j<job->resource_usage[i].npids; j++) {
                json_AddItemToArray(jsonPidArray,
                                    json_CreateNumber(job->resource_usage[i].pidinfo[j].pid));
            }
            if (0 < job->resource_usage[i].npids) {
                json_AddItemToObject(jsonUsageHost, "PID", jsonPidArray);
            }

            if (0 < json_GetArraySize(jsonUsageHost)) {
                json_AddItemToObject(jsonResourceUsage, job->execute_nodes[i], jsonUsageHost);
            }
        }

        json_AddNumberToObject(jsonJobStart, "NUM_ALLOCATE_SLOT", numAllocSlot);
        json_AddNumberToObject(jsonJobStart, "NUM_EXECUTE_NODE", job->num_execute_node);
        json_AddItemToObject(jsonJobStart, "EXECUTE_NODE", jsonNodeArray);

        if (NULL != job->execute_homedir && '\0' != job->execute_homedir[0]) {
            s2j_json_set_basic_element(jsonJobStart, job, string, "EXECUTE_HOMEDIR", execute_homedir);
        }

        if (NULL != job->execute_cwd && '\0' != job->execute_cwd[0]) {
            s2j_json_set_basic_element(jsonJobStart, job, string, "EXECUTE_CWD", execute_cwd);
        }

        if (NULL != job->execute_user && '\0' != job->execute_user[0]) {
            s2j_json_set_basic_element(jsonJobStart, job, string, "EXECUTE_USER", execute_user);
        }

        if (0 < json_GetArraySize(jsonResourceUsage)) {
            json_AddItemToObject(jsonJobStart, "NODE_RESOURCE_USAGE", jsonResourceUsage);
        }

        if (NULL != job->job_GPU && NULL != job->job_GPU->alloc_job_gpu) {
            json *jsonGpuAlloc = json_CreateObject();

            for (i = 0; i < job->num_execute_node; i++) {
                json *jsonGpuHost = json_CreateObject();

                struct jobHostGPUinfo *tmpGpu;
                tmpGpu = &(job->job_GPU->alloc_job_gpu[i]);

                unsigned int j;
                for (j = 0; j < tmpGpu->num_gpu; j++) {

                    json_AddNumberToObject(jsonGpuHost, "GPU_ID", tmpGpu->gpus[j].index);
                    json_AddStringToObject(jsonGpuHost, "GPU_MODEL", tmpGpu->gpus[j].model);

                    if (!FLOAT_EQUAL_ZERO(tmpGpu->gpus[j].alloc_mem)) {
                        double memory;
                        char *strUnit;
                        if (1024*1024 < tmpGpu->gpus[j].alloc_mem) {
                            memory = (double)(tmpGpu->gpus[j].alloc_mem)/(double)(1024*1024);
                            strUnit = (char *)"T";
                        } else if (1024 < tmpGpu->gpus[j].alloc_mem) {
                            memory = (double)(tmpGpu->gpus[j].alloc_mem)/(double)(1024);
                            strUnit = (char *)"G";
                        } else {
                            strUnit = (char *)"M";
                            memory = tmpGpu->gpus[j].alloc_mem;
                        }

                        char strMem[56+1];
                        snprintf(strMem, 56, "%.1f%s", memory, strUnit);
                        json_AddStringToObject(jsonGpuHost, "GPU_MEM", strMem);
                    }
                    json_AddNumberToObject(jsonGpuHost, "GPU_FACTOR", tmpGpu->gpus[j].factor);
                } 

                json_AddItemToObject(jsonGpuAlloc, job->execute_nodes[i], jsonGpuHost);
            }

            json_AddItemToObject(jsonJobStart, "GPU_ALLOCATION", jsonGpuAlloc);
        }

        if (NULL != job->job_numa && NULL != job->job_numa->alloc_job_numa) {
            json *jsonJobNuma = json_CreateObject();

            for (i = 0; i < job->num_execute_node; i++) {
                json *jsonNumaHost = json_CreateObject();

                struct jobHostUsedcore *jobCores;
                jobCores = &(job->job_numa->alloc_job_numa[i]);

                char *strBindType=NULL;
                if (job->job_numa->bind_type == BIND_TYPE_NUMA) {
                    strBindType =  (char *)"numa";
                } else if (job->job_numa->bind_type == BIND_TYPE_SOCKET) {
                    strBindType =  (char *)"socket";
                } else if (job->job_numa->bind_type == BIND_TYPE_CORE) {
                    strBindType =  (char *)"core";
                } else if (job->job_numa->bind_type == BIND_TYPE_THREAD) {
                    strBindType =  (char *)"thread";
                } 
                if (NULL != strBindType) {
                    json_AddStringToObject(jsonNumaHost, "BIND_TYPE", strBindType);
                }

                char *strSame=NULL;
                if (job->job_numa->same == BIND_TYPE_NUMA) {
                    strSame =  (char *)"numa";
                } else if (job->job_numa->same == BIND_TYPE_SOCKET) {
                    strSame =  (char *)"socket";
                } else if (job->job_numa->same == BIND_TYPE_CORE) {
                    strSame =  (char *)"core";
                } else if (job->job_numa->same == BIND_TYPE_THREAD) {
                    strSame =  (char *)"thread";
                }  
                if (NULL != strSame) {
                    json_AddStringToObject(jsonNumaHost, "SAME_LEVEL", strSame);
                }

                char *strExclusive=NULL;
                if (job->job_numa->exclusive == BIND_TYPE_NUMA) {
                    strExclusive =  (char *)"numa";
                } else if (job->job_numa->exclusive == BIND_TYPE_SOCKET) {
                    strExclusive =  (char *)"socket";
                } else if (job->job_numa->exclusive == BIND_TYPE_CORE) {
                    strExclusive =  (char *)"core";
                } else if (job->job_numa->exclusive == BIND_TYPE_THREAD) {
                    strExclusive =  (char *)"thread";
                }  
                if (NULL != strExclusive) {
                    json_AddStringToObject(jsonNumaHost, "EXCLUSIVE", strExclusive);
                }

                HASH_TABLE_T *hashNuma = gf_hash_make(LIST_NUM_ENTS(jobCores->used_numa));
                HASH_TABLE_T *hashSocket = gf_hash_make(LIST_NUM_ENTS(jobCores->used_numa));

                struct usedCoreInfoDown *tmp = NULL;
                for (tmp = (struct usedCoreInfoDown *)jobCores->used_numa->back;
                     tmp != (struct usedCoreInfoDown *)jobCores->used_numa;
                     tmp = (struct usedCoreInfoDown *)tmp->back) {

                    char affinityIndex[56+1];
                    char key[12];
                    if (job->job_numa->bind_type == BIND_TYPE_NUMA) {

                        snprintf(key, 12, "%d", tmp->pparent_index);
                        if (NULL != gf_hash_find(hashNuma, key)) {
                            continue;
                        }

                        snprintf(affinityIndex, 56, "%d", tmp->pparent_index);
                        gf_hash_install(hashNuma, key, tmp);
                    } else if (job->job_numa->bind_type == BIND_TYPE_SOCKET) {

                        snprintf(key, 12, "%d", tmp->parent_index);
                        if (NULL != gf_hash_find(hashSocket, key)) {
                            continue;
                        }

                        snprintf(affinityIndex, 56, "%d/%d",
                                 tmp->pparent_index, tmp->parent_index);
                        gf_hash_install(hashSocket, key, tmp);
                    } else {
                        snprintf(affinityIndex, 56, "%d/%d/%d",
                                 tmp->pparent_index, tmp->parent_index, tmp->index_sibling);
                    }

                    json_AddStringToObject(jsonNumaHost, "INDEX", affinityIndex);
                }
                gf_hash_free(hashNuma, gf_free_none);
                gf_hash_free(hashSocket, gf_free_none);

                json_AddItemToObject(jsonJobNuma, job->execute_nodes[i], jsonNumaHost);
	    }

            json_AddItemToObject(jsonJobStart, "AFFINITY", jsonJobNuma);
        }

        if (0 < totalCpuTime) {
            char strCpuTime[24+1];
            snprintf(strCpuTime, 24, "%ld", totalCpuTime);
            json_AddStringToObject(jsonJobStart, "TOTAL_CPU_TIME", strCpuTime);
        }

        if (0 < totalMem) {
            char *strUnit;
            float memVal;

            if (totalMem > 1024*1024*1000) {
                memVal = (double)totalMem/(double)(1024*1024*1000);
                strUnit = (char *)"T";
            } else if (totalMem > 1024*1024) {
                memVal = (double)totalMem/(double)(1024*1024);
                strUnit = (char *)"G";
            } else if (totalMem > 1024) {
                memVal = (double)totalMem/(double)(1024);
                strUnit = (char *)"M";
            } else {
                memVal = totalMem;
                strUnit = (char *)"K";
            }

            char strMemUsage[56+1];
            snprintf(strMemUsage, 56, "%.1f%s", memVal, strUnit);
            json_AddStringToObject(jsonJobStart, "TOTAL_MEMORY_USAGE", strMemUsage);
        }

        if (0 < totalSwap) {
            char *strUnit;
            float swapVal;

            if (totalSwap > 1024*1024*1024) {
                swapVal = (double)totalSwap/(double)(1024*1024*1024);
                strUnit = (char *)"T";
            } else if (totalSwap > 1024*1024) {
                swapVal = (double)totalSwap/(double)(1024*1024);
                strUnit = (char *)"G";
            } else if (totalSwap > 1024) {
                swapVal = (double)totalSwap/(double)1024;
                strUnit = (char *)"M";
            } else {
                swapVal = totalSwap;
                strUnit = (char *)"K";
            }

            char strSwapUsage[56+1];
            snprintf(strSwapUsage, 56, "%.1f%s", swapVal, strUnit);
            json_AddStringToObject(jsonJobStart, "TOTAL_SWAP_USAGE", strSwapUsage);
        }

        char *suspendReasons;
        switch (job->status) {
        case JOB_STATUS_SSUSP:
        case JOB_STATUS_USUSP:

            if (0 < job->reason_id) {
                suspendReasons = get_suspend_reason(job->reason_id, job->reason_detail,
                                                    info->resource_table);
                json_AddStringToObject(jsonJobStart, "SUSPEND_REASON", suspendReasons);
            }

            break;
        }

        json_AddItemToObject(jsonJob, "START_INFO", jsonJobStart);
    }

    if (0 < job->end_time) {
        json *jsonJobFinish = json_CreateObject();

        ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &(job->end_time), timebuf);
        json_AddStringToObject(jsonJobFinish, "FINISH_TIME", timebuf);

        switch (job->status) {
        case JOB_STATUS_EXIT:

            json_AddTrueToObject(jsonJobFinish, "JOB_EXIT");
            break;
        case JOB_STATUS_DONE:
        case JOB_STATUS_DONE|JOB_STATUS_PDONE:
        case JOB_STATUS_DONE|JOB_STATUS_PERR:

            json_AddTrueToObject(jsonJobFinish, "JOB_DONE");
            break;
        }

        STATUS_WAIT_T wStatus;
        JOB_WSTATUS(wStatus) = job->exit_stat;
        if (0 != job->exit_stat) {
            if (WEXITSTATUS(wStatus)) {
                json_AddNumberToObject(jsonJobFinish, "EXIT_CODE", WEXITSTATUS(wStatus));
            } else {
                json_AddNumberToObject(jsonJobFinish, "EXIT_SIGNAL", WTERMSIG(wStatus));
            }
        }

        if (!FLOAT_EQUAL_ZERO(job->used_cpu_time)) {

            char strCpuTime[MAX_TIMERSTRLEN];
            cputimer_2string(job->used_cpu_time, strCpuTime);
            json_AddStringToObject(jsonJobFinish, "CPU_TIME", strCpuTime);
        }

        if (0 < job->max_mem) {
            char *strUnit;
            float memVal;

            if (job->max_mem > 1024*1024*1024) {
                memVal = (double)job->max_mem/(double)(1024*1024*1024);
                strUnit = (char *)"T";
            } else if (job->max_mem > 1024*1024) {
                memVal = (double)job->max_mem/(double)(1024*1024);
                strUnit = (char *)"G";
            } else if (job->max_mem > 1024) {
                memVal = (double)job->max_mem/(double)(1024);
                strUnit = (char *)"M";
            } else {
                memVal = job->max_mem;
                strUnit = (char *)"K";
            }

            char strMemUsage[56+1];
            snprintf(strMemUsage, 56, "%.1f%s", memVal, strUnit);
            json_AddStringToObject(jsonJobFinish, "MEMORY_MAX", strMemUsage);
            //json_AddStringToObject(jsonJobFinish, "MEMORY_AVG", strMemUsage);
        }

        json_AddItemToObject(jsonJob, "END_INFO", jsonJobFinish);
    }

    return jsonJob;
} // end function jobinfo_2json

/**
 * @brief       check if I am cluster administrator or not.
 *
 * @retval      TRUE          #1: I'm cluster admin
 * @retval      FALSE         #2: I'm not
 */
static bool_t
isquickpool_admin(void)
{
    char *strMyCluster;
    strMyCluster = qp_getclustername();
    if (NULL == strMyCluster) {
        log_message(ERR, NOCLASS, "%s: qp_getclustername() failed.", __func__);
        return FALSE;
    }

    struct request clusterRequest;
    memset(&clusterRequest, 0, sizeof(struct request));

    clusterRequest.number = 1;
    clusterRequest.keyvalues = (struct keyvalue *)calloc(1, sizeof(struct keyvalue));
    if (NULL == clusterRequest.keyvalues) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__,
                    "calloc", sizeof(struct keyvalue));
        quickpool_errno = ERROR_MEMORY;
        return FALSE;
    }

    clusterRequest.keyvalues[0].key = REQUEST_KEY_OPTIONS;
    clusterRequest.keyvalues[0].value = (char *)calloc(12, sizeof(char));
    if (NULL == clusterRequest.keyvalues[0].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
        quickpool_errno = ERROR_MEMORY;
        return FALSE;
    }
    snprintf(clusterRequest.keyvalues[0].value, 12, "%d", 0);

    struct QuickPoolClusterReply *clusterReply;
    clusterReply = qp_clusterinfo(&clusterRequest, NULL);
    if (quickpool_errno != 0 || clusterReply == NULL) {

        if (clusterReply != NULL) {
            log_message(ERR, NOCLASS, "%s: fail to get cluster info due to %s.",
                        __func__, clusterReply->error_message);
        }

        FREEUP(strMyCluster);
        gf_xdr_free(gf_xdr_request, &clusterRequest);

        return FALSE;
    }
    gf_xdr_free(gf_xdr_request, &clusterRequest);

    char *strMyName;
    strMyName = getmy_uname();

    struct QuickPoolClusterInfo *clusterInfo;
    clusterInfo = clusterReply->clusters;

    unsigned int i, j;
    for (i = 0; i < clusterReply->num_clusters; i++) {

        if (0 != strcmp(strMyCluster, clusterInfo[i].cluster_name)) {
            continue;
        }

        for (j = 0; j < clusterInfo->num_admins; j++) {

            if (strcmp(strMyName, clusterInfo->admins[j]) == 0) {

                FREEUP(strMyName);
                FREEUP(strMyCluster);
                return TRUE;
            }
        }
    }

    FREEUP(strMyName);
    FREEUP(strMyCluster);
    gf_xdr_free(libxdr_clusterinfo_reply, clusterReply);
    FREEUP(clusterReply);

    return FALSE;
} // end function isquickpool_admin

static void
listjob_usage(const char *cmd)
{

    fprintf(stdout, "qp job %s usage: [-h] [-a] [-d] [-p] [-s] [-f] [-w] [-r] [-g job_group]\n", cmd);
    fprintf(stdout, "\t\t   [-A] [-m host_name] [-q queue_name] [-u user_name | -u all]\n");
    fprintf(stdout, "\t\t   [-P project_name] [-J name_spec]\n");
    fprintf(stdout, "\t\t   [-gpu]\n");
    fprintf(stdout, "\t\t   [jobId | \"jobId[idxList]\" ...]\n");

    return;
}
